#pragma once
#include <iostream>
#include <string>
#include <fstream>
#include <boost/algorithm/string.hpp>
#include "cppjieba/Jieba.hpp"
//#include <codecvt>
//#include <locale>

namespace ns_util
{
    class FileUtil
    {
    public:
        static bool ReadFile(const std::string& file_path,std::string& out)
        {
            //std::cout<<"开始读取文件"<<std::endl;
            std::ifstream in(file_path,std::ios::in);
            if(!in.is_open())
            {
                std::cerr<<file_path<<" open error!"<<std::endl;
                return false;
            }
            //文件打开成功
            std::string line;
            while(std::getline(in,line))
            {
                out+=line;
            }

            in.close();
            return true;
        }
    };
    class StringUtil
    {
    public:
        static void Split(const std::string& line,std::vector<std::string>& results,const std::string& sep)
        {
            //boost->split
            boost::split(results,line,boost::is_any_of(sep),boost::token_compress_on);
        }
    };

    const char* const DICT_PATH = "./dict/jieba.dict.utf8";
    const char* const HMM_PATH = "./dict/hmm_model.utf8";
    const char* const USER_DICT_PATH = "./dict/user.dict.utf8";
    const char* const IDF_PATH = "./dict/idf.utf8";
    const char* const STOP_WORD_PATH = "./dict/stop_words.utf8";

    // 把非法 UTF-8 序列剔除，返回合法字符串
    // std::string SanitizeUtf8(const std::string& raw)
    // {
    //     std::string out;
    //     out.reserve(raw.size());
    //     for (size_t i = 0; i < raw.size(); )
    //     {
    //         unsigned char c = raw[i];
    //         if (c < 0x80) { out += c; ++i; }                       // 1-byte
    //         else if ((c & 0xE0) == 0xC0) {                         // 2-byte
    //             if (i + 1 < raw.size() && (raw[i + 1] & 0xC0) == 0x80)
    //                 { out.append(raw, i, 2); i += 2; }
    //             else ++i;
    //         }
    //         else if ((c & 0xF0) == 0xE0) {                         // 3-byte
    //             if (i + 2 < raw.size() &&
    //                 (raw[i + 1] & 0xC0) == 0x80 &&
    //                 (raw[i + 2] & 0xC0) == 0x80)
    //                 { out.append(raw, i, 3); i += 3; }
    //             else ++i;
    //         }
    //         else if ((c & 0xF8) == 0xF0) {                         // 4-byte
    //             if (i + 3 < raw.size() &&
    //                 (raw[i + 1] & 0xC0) == 0x80 &&          
    //                 (raw[i + 2] & 0xC0) == 0x80 &&
    //                 (raw[i + 3] & 0xC0) == 0x80)
    //                 { out.append(raw, i, 4); i += 4; }
    //             else ++i;
    //         }
    //         else ++i;                                              // 非法头，跳过
    //     }
    //     return out;
    // }
  
    class JiebaUtil
    {
    private:
        static cppjieba::Jieba jieba;
    private:
       
    public:
        static void CutString(const std::string&src,std::vector<std::string>*out)
        {
            // 1. 先清洗
            //std::string clean = SanitizeUtf8(src);
            //2.再分词
             jieba.CutForSearch(src,*out);
            //jieba.CutForSearch(clean,*out);
        }
    };
    cppjieba::Jieba JiebaUtil::jieba(DICT_PATH, HMM_PATH, USER_DICT_PATH, IDF_PATH, STOP_WORD_PATH);
}
